<?php
/**
 * 数据过滤
 */
ini_set('memory_limit', '4024M');
header("content-type:text/plain;charset=utf8");
require './include/CurlMulti.php';
require './include/config.php';
// 小区列表
global $XIAOQU_LIST;
global $url;
// 城市URL列表
global $CITY_URL;
// 关键词
global $WD; 
$limit = '5000';
$page = 0;
$max_page = 1;
$size = 5000;
$last_id = 0;
while(true) {
    $page++;
    $WD = getDiffxiaoqu();
    //$WD = getXiaoquKeyword(110000, 5000, $last_id);
    if (!$WD || $page > $max_page) {
        break;
    }
    echo count($WD), "\n";
    isset($_GET['wd']) && $WD = explode(",",trim($_GET['wd']));
    // 分页每页数据条数
    global $RN; 
    $RN = 50;
    
    // 获取城市
    global $CITY;
    $CITY = array('北京市');
    // qt 查询数据类型   --- s 城市列表   sport 城市关键词查询   bsi 地铁查询
    $url = 'http://map.baidu.com/?qt=spot&l=16';
    
    // $param = array('wd'=>$WD,'c'=>'city_code_int', 'nn'=>'偏移量');
    
    $CurlMulti = new CurlMulti(); 
    $CurlMulti->cache = array (
    	'dir' => 'E:\clone\baidu\filter',
    	'on' => true,
    	'expire' => 3600 * 2400,
        'dirLevel' => 2
    );
    $CurlMulti->maxThread = 50;
    // 对所有任务有效，可以被add()参数中的opt覆盖。
    $CurlMulti->opt[CURLOPT_CONNECTTIMEOUT] = 100;
    // $CurlMulti->opt[CURLOPT_FTPPORT] = '58.63.237.151';
    
    
    $nn = 0;
    foreach ((array)$WD as $id => $w) {
        if (is_array($w)) {
            $wd = $w['name'];
            $city = array(
                'name'=>$w['city'],
                'code'=>$w['code'],
            );
        } else {
            $wd = $w;
            $city = array(
                'name'=>'北京',
                'code'=>'131',
            );
        }
        echo $wd ,"\t",$city['name'], "\n";
        $param = array('wd'=>$wd,'c'=>$city['code'], 'nn'=>$nn, 'rn'=>$RN);
        $CITY_URL[$city['name'] .'-'. $wd .'-'. $nn] = array(
            'name' => $city['name'],
            'code' => $city['code'],
            'url' => $url . '&' . urldecode(http_build_query($param)) .'&v=1',
            'curl' => false,
            'wd' => $wd,
        );
        $last_id = $id;
    }
    
    // 获取每个城市中 所有小区列表 获取分页链接
    foreach ($CITY_URL as $city => $item) {
        $CITY_URL[$city]['curl'] = true;
        $CurlMulti->add(array(
            'url' => $item['url'],
            'args'=> $item,
        ), function ($r, $args){
            global $CITY_URL, $url, $WD, $RN;
            $contents = json_decode($r['content'], true);
            isset($contents['result']['aladdin_res_num']) && $contents['result']['aladdin_res_num'] = 0;
            for ($nn=1; $nn<$contents['result']['aladdin_res_num']; $nn+=$RN) {
                $param = array('wd'=>$args['wd'],'c'=>$args['code'], 'nn'=>$nn, 'rn'=>$RN);
                $CITY_URL[$args['name'] .'-'. $args['wd'] .'-'. $nn] = array(
                    'name' => $args['name'],
                    'code' => $args['code'],
                    'url' => $url . '&' . urldecode(http_build_query($param)),
                    'curl' => false,
                );
            }
            
            
            // 添加到数据 列表中
            global $XIAOQU_LIST;
            if (!isset($contents['content']) || !is_array($contents['content'])) {
                return false;
            }
            foreach ($contents['content'] as $item) {
                foreach ($args as $k => $v) {
                    $item['city_'.$k] = $v;
                }
                $cla = array();
                foreach ($item['cla'] as $k => $v) {
                    $cla[] = array('cla_id'=> $v[0], 'cla_name'=> $v[1]);
                }
                $item['cla'] = $cla;
                $XIAOQU_LIST[$args['name'] . $item["name"]] = $item;
            }
            
        }, function ($r, $args){
            var_dump($r, $args);
        });
    }
    
    $CurlMulti->start();
    echo 'line:', __LINE__ , ' $CITY_URL:' , count($CITY_URL), "\n";
    
    
    // 获取每个城市中 每个小区详细信息
    foreach ($CITY_URL as $city => $item) {
        if ($CITY_URL[$city]['curl']) {
            continue;
        }
        $CITY_URL[$city]['curl'] = true;
        $url = $item['url'];
        $CurlMulti->add(array(
            'url' => $url,
            'args'=> $item
        ), function ($r, $args){
            global $XIAOQU_LIST, $CITY_URL;
            $contents = json_decode($r['content'], true);
            if (!isset($contents['content']) || !is_array($contents['content'])) {
                return false;
            }
            foreach ($contents['content'] as $item) {
                foreach ($args as $k => $v) {
                    $item['city_'.$k] = $v;
                }
                $cla = array();
                foreach ($item['cla'] as $k => $v) {
                    $cla[] = array('cla_id'=> $v[0], 'cla_name'=> $v[1]);
                }
                $item['cla'] = $cla;
                $XIAOQU_LIST[$item["city_name"] . $item["name"]] = $item;
            }
        }, function ($r, $args){
            var_dump($r, $args);
        });
    }
    
    $CurlMulti->start();
    
    
    echo 'line:', __LINE__ , ' 小区数量:' , count($XIAOQU_LIST), "\n";
    
    // 存储数据
    
    $index = 'baidu_xiaoqu';
    $type = 'xiaoqu';
    
    $es_bulk = $conf['es']['hosts'].'/'.$index.'/'.$type.'/_bulk';
    $es_url = $conf['es']['hosts'].'/'.$index.'/'.$type.'/';
    $request = array();
    $times = 0;
    
    $body = '';
    foreach($XIAOQU_LIST as $key => $item){
        echo $key, "\n";
        $_index = array("index"=>array("_id"=>sprintf("%u",crc32($key))));
        $body .= json_encode($_index) . "\n";
        $body .=  json_encode($item) . "\n";
        $times++;
        if (($times+1) % 1000 == 0) {
            $multi_request[] = $body;
            $body = "";
        }
    }
    $multi_request[] = $body;
    global $request;
    $request = 0;
    try {
        foreach ($multi_request as $body) {
            $res = fly_curl($es_bulk,$body);
            $request++;
        }
    } catch(Exception $e) {
        echo '<pre>';
        var_dump($e->getCode(), $e->getLine(), $e->getMessage());
        echo '</pre>';
    }
    
    echo $request, "\n";
}
